{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "99856bdb-13b9-4f4d-b4c3-28a844c4ac00",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cc4c5652-2569-4c85-9019-37e724b4a0e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "36377dd0-f411-42dc-8dde-fdbf46700353",
   "metadata": {},
   "outputs": [],
   "source": [
    "file_lis = []\n",
    "for i in os.listdir('weibo_data/'):\n",
    "    if not os.path.isdir(os.path.join('weibo_data', i)):\n",
    "        continue\n",
    "    else:\n",
    "        j = os.path.join('weibo_data', i)\n",
    "        file_lis.append(os.path.join(j, os.listdir(os.path.join('weibo_data', i))[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "82da0e2f-a3b4-456a-9b01-4a17c2bba720",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame()\n",
    "for i in file_lis:\n",
    "    df = pd.concat([df, pd.read_csv(i, low_memory=False)], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4bbdf4e3-5db9-4262-9af8-164ac9ffb7e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop_duplicates(subset='bid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8a9e01bf-ebab-4329-b3ae-258a063ea4ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "517c22d6-920b-4199-b5e8-db90c13fc427",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "l = pd.read_table('user_id.txt', header=None)[0].to_list()+pd.read_table('user_id_copy1.txt', header=None)[0].to_list()+pd.read_table('user_id_copy2.txt', header=None)[0].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9af68898-8dc4-403f-8714-787961d6d7ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in df.user_id.unique():\n",
    "    l.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "02610099-fd48-4f87-b370-699455e160a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_id_copy1 = []\n",
    "for item in Counter(l).most_common():\n",
    "    if item[1] == 1:\n",
    "        user_id_copy1.append(item[0])\n",
    "    else:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9592f189-c909-40f0-abf1-6840ec9ccbea",
   "metadata": {},
   "outputs": [],
   "source": [
    "txt_file = 'user_id_copy3.txt'\n",
    "f = open(txt_file, \"a+\", encoding='utf8')\n",
    "for j in user_id_copy1:\n",
    "    f.write('{}\\n'.format(j))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "91a07f50-1f11-4ded-a45c-d60b78822005",
   "metadata": {},
   "outputs": [],
   "source": [
    "l = pd.read_table('bid_id.txt', header=None)[0].to_list()+pd.read_table('bid_id_copy1.txt', header=None)[0].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5e006875-2cfc-44ae-be4a-c4c18aaa2520",
   "metadata": {},
   "outputs": [],
   "source": [
    "l = list(set(l))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c2ae8c90-9819-4b5c-9342-1048a7c294c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in df[df[\"评论数\"] > 0][\"bid\"].to_list():\n",
    "    l.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b2ce8a36-92f0-4ee8-abb9-dc65d88259e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "bid_id_copy1 = []\n",
    "for item in Counter(l).most_common():\n",
    "    if item[1] == 1:\n",
    "        bid_id_copy1.append(item[0])\n",
    "    else:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0ae98e23-7c63-411b-9f12-05e91a7ce53c",
   "metadata": {},
   "outputs": [],
   "source": [
    "txt_file = 'bid_id_copy2.txt'\n",
    "f = open(txt_file, \"a+\", encoding='utf8')\n",
    "for j in bid_id_copy1:\n",
    "    f.write('{}\\n'.format(j))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0974f45b-d77b-473b-bdb6-71111237b611",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4683945"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df[\"评论数\"] > 0][\"评论数\"].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a794f91a-e8b7-4096-90c5-39ffc9f04fa1",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c4549965-2203-46dc-a5a3-b35615ac23a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a4cdb8a0-1242-4650-97c2-c9cfbb24f2e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0caa9656-53ec-4ee0-b8a1-1d6bcd3cdd8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame()\n",
    "for i in os.listdir('weibo_data/'):\n",
    "    if i.endswith('jsonl'):\n",
    "        data = pd.concat([data, pd.read_table(os.path.join('weibo_data', i), header=None)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d335c24e-3164-46c2-856d-52fd3f08e769",
   "metadata": {},
   "outputs": [],
   "source": [
    "RES = []\n",
    "for item in data[0]:\n",
    "    RES.append(json.loads(item))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "129d5f1f-ad40-44f7-94af-decd2540ccb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "extra = []\n",
    "for i in RES:\n",
    "    extra.append({\"author_name\":i[\"comment_user\"][\"nick_name\"], \n",
    "                  \"author_desc\":i[\"comment_user\"][\"description\"], \n",
    "                  \"followers_count\": i[\"comment_user\"][\"followers_count\"], \n",
    "                  \"friends_count\": i[\"comment_user\"][\"friends_count\"], \n",
    "                  \"gender\": i[\"comment_user\"][\"gender\"], \n",
    "                  \"location\": i[\"comment_user\"][\"location\"]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c0b23e32-18ca-4538-8ea0-8b8f65fe8a50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author_name</th>\n",
       "      <th>author_desc</th>\n",
       "      <th>followers_count</th>\n",
       "      <th>friends_count</th>\n",
       "      <th>gender</th>\n",
       "      <th>location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11小花1</td>\n",
       "      <td></td>\n",
       "      <td>246</td>\n",
       "      <td>950</td>\n",
       "      <td>f</td>\n",
       "      <td>广东 深圳</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18老妖</td>\n",
       "      <td>会哭，会笑，挺真实一Y头</td>\n",
       "      <td>143</td>\n",
       "      <td>419</td>\n",
       "      <td>f</td>\n",
       "      <td>福建 厦门</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>哒滴哒滴哒哒滴哒</td>\n",
       "      <td>老兵</td>\n",
       "      <td>83</td>\n",
       "      <td>397</td>\n",
       "      <td>m</td>\n",
       "      <td>北京</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>南乔在线1314</td>\n",
       "      <td>爱笑的女子运气都不会太差</td>\n",
       "      <td>138</td>\n",
       "      <td>594</td>\n",
       "      <td>f</td>\n",
       "      <td>四川 成都</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>hlslxy1314</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>m</td>\n",
       "      <td>其他</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1439292</th>\n",
       "      <td>一战倾心z</td>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>45</td>\n",
       "      <td>f</td>\n",
       "      <td>其他</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1439293</th>\n",
       "      <td>醒醒你头发没了</td>\n",
       "      <td>哈哈哈哈哈哈哈哈哈 枇杷</td>\n",
       "      <td>16</td>\n",
       "      <td>117</td>\n",
       "      <td>f</td>\n",
       "      <td>海外</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1439294</th>\n",
       "      <td>锋少199406</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>129</td>\n",
       "      <td>m</td>\n",
       "      <td>其他</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1439295</th>\n",
       "      <td>走走看看逛逛蹓跶蹓跶</td>\n",
       "      <td>知识是力量，良知引方向！</td>\n",
       "      <td>207</td>\n",
       "      <td>486</td>\n",
       "      <td>m</td>\n",
       "      <td>陕西</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1439296</th>\n",
       "      <td>笑口榛子88278</td>\n",
       "      <td>只是一个小小散。</td>\n",
       "      <td>1096</td>\n",
       "      <td>888</td>\n",
       "      <td>m</td>\n",
       "      <td>其他</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1439297 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        author_name   author_desc  followers_count  friends_count gender  \\\n",
       "0             11小花1                            246            950      f   \n",
       "1              18老妖  会哭，会笑，挺真实一Y头              143            419      f   \n",
       "2          哒滴哒滴哒哒滴哒            老兵               83            397      m   \n",
       "3          南乔在线1314  爱笑的女子运气都不会太差              138            594      f   \n",
       "4        hlslxy1314                              0             50      m   \n",
       "...             ...           ...              ...            ...    ...   \n",
       "1439292       一战倾心z                              6             45      f   \n",
       "1439293     醒醒你头发没了  哈哈哈哈哈哈哈哈哈 枇杷               16            117      f   \n",
       "1439294    锋少199406                              1            129      m   \n",
       "1439295  走走看看逛逛蹓跶蹓跶  知识是力量，良知引方向！              207            486      m   \n",
       "1439296   笑口榛子88278      只是一个小小散。             1096            888      m   \n",
       "\n",
       "        location  \n",
       "0          广东 深圳  \n",
       "1          福建 厦门  \n",
       "2             北京  \n",
       "3          四川 成都  \n",
       "4             其他  \n",
       "...          ...  \n",
       "1439292       其他  \n",
       "1439293       海外  \n",
       "1439294       其他  \n",
       "1439295       陕西  \n",
       "1439296       其他  \n",
       "\n",
       "[1439297 rows x 6 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(extra)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "77056d7f-9864-4535-bc12-b91c13fb2bc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df = pd.concat([pd.DataFrame(RES).drop([\"comment_user\",\"crawl_time\",\"_id\"], axis=1), pd.DataFrame(extra)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "073dedd3-a8e4-4192-9181-d647dc5e5c33",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.drop_duplicates(inplace=True, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "58bd7de4-d006-4e10-a985-9e2b6457df29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1407812, 10)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e3e6eae1-21ed-4ed0-a6fd-ac46c8dfd2fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.to_csv('weibo_data/comment.csv', encoding='utf-8-sig')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a38544f-04b7-4ee0-9a68-b68e8afeb929",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5ca9e83c-6967-4b7c-9162-cedc7238b3b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ee8127ea-9cf4-416b-bab5-9f3c701b4fb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('weibo_data/抗疫/抗疫.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "4105816d-fe38-486b-bafb-5dbc69bfdc16",
   "metadata": {},
   "outputs": [],
   "source": [
    "inx_ = []\n",
    "for inx, v in enumerate(data[\"微博正文\"]):\n",
    "    if '抗疫' in v:\n",
    "        inx_.append(inx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "75f7eb32-9973-4ce4-b83f-bf8820932b5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.loc[inx_, :].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "4d5742c7-4009-4a24-9b90-4ab63cae1c72",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "l = pd.read_table('bid_id.txt', header=None)[0].to_list()+pd.read_table('bid_id_copy1.txt', header=None)[0].to_list()+pd.read_table('bid_id_copy2.txt', header=None)[0].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6cd674e2-e38b-4921-862a-26ee2d8b3170",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "c18c664b-a1b9-45d8-90a1-82e61fd5ba67",
   "metadata": {},
   "outputs": [],
   "source": [
    "ll = []\n",
    "for i in data[data[\"评论数\"] > 0][\"bid\"].to_list():\n",
    "    ll.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "3f6a2576-6d05-4e62-8c29-2a9cc5caf03e",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_ = list(set(l))+list(set(ll))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "b49b0bee-1c51-4d01-860f-e7f374a8e5f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "flag = []\n",
    "for item in Counter(all_).most_common():\n",
    "    if item[1] == 2:\n",
    "        flag.append(item[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "0b796a56-7df0-4403-8b06-314a9394c91c",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = []\n",
    "for i in Counter(flag+list(set(ll))).most_common():\n",
    "    if i[1] == 1:\n",
    "        a.append(i[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "2e7346f8-87e1-44c3-b70c-13f6ebf0b756",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "txt_file = 'bid_id_copy3.txt'\n",
    "f = open(txt_file, \"a+\", encoding='utf8')\n",
    "for j in a:\n",
    "    f.write('{}\\n'.format(j))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ddbfa95-f34b-467a-8701-da2799c3d407",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "363ad174-1b0a-410a-8057-cc002c4ef399",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sklearn",
   "language": "python",
   "name": "sklean"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
